package com.freez.spark.ml;

import org.apache.spark.ml.linalg.VectorUDT;
import org.apache.spark.ml.linalg.Vectors;
import org.apache.spark.ml.stat.ChiSquareTest;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.DataTypes;
import org.apache.spark.sql.types.Metadata;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;

import java.util.Arrays;
import java.util.List;

/**
 * FREEDOM  2021 人生苦短，不妨一试
 *
 * @FileName: ChiSquare.java
 * @Author: zcs
 * @Date: 2021年-12月-07日 周二 18:37
 * @Description: 卡方检测
 */
public class ChiSquare {
    /**
     * 卡方检测表示统计样本的实际观测值和预测值之间的偏离程度，</br>
     * 实际观测值与预测值之间的偏离程度决定卡方值的大小，</br>
     * 卡方值越大，表示越偏离样本的实际值，反之，越小表示越接近实际值，</br>
     * 如果卡方为0，表示预测值和实际值完全吻合 </br>
     * <p>
     * pValues：评测值，越大【接近1】代表该特征列对标签的区分作用越低，反之，越小【接近0】越有区分价值。 </br>
     * degreeOfFreedom：自由度，degreeOfFreedom+1等价于该特征值的种类。 </br>
     * statistics：处理逻辑比较复杂，可以认为是越大分类价值越高，越小分类价值越低。</br>
     */
    public static void main(String[] args) {
        SparkSession sparkSession = SparkSession.builder().appName("SparkSQLTest").master("local").getOrCreate();
        List<Row> list = Arrays.asList(RowFactory.create(1, Vectors.dense(1), 1, "zcs"),
                RowFactory.create(Vectors.dense(2.0, 2.0), 2, "小华"),
                RowFactory.create(3.0, Vectors.dense(3.0, 3.0), 3, "zcs"),
                RowFactory.create(3.0, Vectors.dense(3.0, 3.0), 4, "zcs"),
                RowFactory.create(2.0, Vectors.dense(2.0, 2.0), 5, "捞的猪"),
                RowFactory.create(1.0, Vectors.dense(1, 2), 6, "王德发"));
        StructType structType = new StructType(
                new StructField[]{new StructField("num_label", DataTypes.IntegerType, false, Metadata.empty()),
                        new StructField("vector_feature", new VectorUDT(), false, Metadata.empty()),
                        new StructField("id", DataTypes.IntegerType, false, Metadata.empty()),
                        new StructField("name", DataTypes.StringType, false, Metadata.empty())});
        Dataset<Row> mineDataset = sparkSession.createDataFrame(list, structType);
        System.out.println("==============================================");
        // 实际列、标准列
        //Row test = ChiSquareTest.test(mineDataset, "vector", "num").head();
        mineDataset.show(false);
        System.out.println("==============================================");
        Dataset<Row> result = ChiSquareTest.test(mineDataset, "vector_feature", "num_label");
        result.show(false);
        sparkSession.close();
    }
}